# author = zhkai
# date = 2020/4/4

import requests
from redis import StrictRedis
from scrapy.selector import Selector
from lxml import etree
import random
import time

headers = {
    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
    'Cookie':'channelid=0; sid=1585979260404255; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1585979261; _ga=GA1.2.1000868130.1585979261; _gid=GA1.2.540704127.1585979261; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1585979365',
}

redis = StrictRedis(host='localhost', port=6379, db=1)
proxy_key = "proxy_xici"

def get_proxy():
    for i in range(2,4054):
        url = 'https://www.kuaidaili.com/free/inha/{}'.format(i)
        # url = 'https://www.baidu.com'
        response = requests.get(url=url,headers=headers, verify=False)
        print(response.status_code)
        html = etree.HTML(response.text)
        all_trs = html.xpath("//div[@id='list']//tr")

        for tr in all_trs[1:]:
            all_text = tr.xpath("td/text()")
            proxy_ip = all_text[0]
            proxy_port = all_text[1]
            proxy_type = all_text[3]

            proxy_addr = proxy_type+'://' + proxy_ip + ':' +proxy_port
            proxy = {
                proxy_type:proxy_addr
            }
            is_valid = judge_proxy(proxy)
            if is_valid:
                redis.lpush(proxy_key, proxy_addr)
        # num = random.randint(1,3)
        num = random.randint(1,2)
        time.sleep(num)


def judge_proxy(proxy_addr):
    http_url = 'https://www.baidu.com'
    try:

        response = requests.get(url=http_url,headers=headers, proxies = proxy_addr)
        print(response)
        return True
    except Exception as e:
        print('无效代理')
        return False
    else:
        code = response.status_code
        if code>=200 and code < 300:
            return True
        else:
            print('无效代理')
            return False

def del_proxy(value):
    redis.lrem(proxy_key, 0, value)


def get_random_proxy():
    len_proxy = redis.llen(proxy_key)

    index = random.randint(0,len_proxy+1)
    proxy_addr = redis.lindex(proxy_key, index)
    proxy_addr = proxy_addr.decode()
    proxy_type = proxy_addr.split(':')[0] + '://'
    proxy = {
        proxy_type : proxy_addr
    }
    is_valid = judge_proxy(proxy)
    if is_valid:
        return proxy
    else:
        print('无效代理')
        del_proxy(proxy)
        get_random_proxy()

if __name__ == '__main__':
    # get_proxy()
    get_random_proxy()